This notebook is all about benchmarking some R code used in this package.

Hardware / Software used:

  • Intel i7-4600U
  • Compilation flags for C/C++: -O2 -Wall $(DEBUGFLAG) -mtune=core2 (R’s defaults)
  • Windows Server 2012 R2
  • R 3.3.2 + Intel MKL

Libraries

library(data.table)
data.table 1.10.4
  The fastest way to learn (by data.table authors): https://www.datacamp.com/courses/data-analysis-the-data-table-way
  Documentation: ?data.table, example(data.table) and browseVignettes("data.table")
  Release notes, videos and slides: http://r-datatable.com
library(microbenchmark)
library(Rcpp)
library(ggplot2)
library(plotly)

Attaching package: <U+393C><U+3E31>plotly<U+393C><U+3E32>

The following object is masked from <U+393C><U+3E31>package:ggplot2<U+393C><U+3E32>:

    last_plot

The following object is masked from <U+393C><U+3E31>package:stats<U+393C><U+3E32>:

    filter

The following object is masked from <U+393C><U+3E31>package:graphics<U+393C><U+3E32>:

    layout
# Helper function to print data well in tables
print_well <- function(data, digits = 6) {
  
  # To milliseconds
  data <- data / 1000000
  
  # Sprintf helper
  sprintf_helper <- paste0("%.0", digits, "f")
  
  cat("| Min | 25% | 50% | 75% | Max | Mean |  \n| --: | --: | --: | --: | --: | --: |  \n| ", sprintf(sprintf_helper, min(data)), " | ", sprintf(sprintf_helper, quantile(data, probs = 0.25)), " | ", sprintf(sprintf_helper, median(data)), " | ", sprintf(sprintf_helper, quantile(data, probs = 0.75)), " | ", sprintf(sprintf_helper, max(data)), " | ", sprintf(sprintf_helper, mean(data)), " |  \n", sep = "")
  
  return(data)
  
}
# Test case function
# Arguments renamed to avoid recursive clash
test_case <- function(f, preds, labels, eps) {
  cat("Test case: ", do.call(f, list(preds = preds[1:50],
                                     labels = labels[1:50],
                                     eps = 1e-15)), "  \n", sep = "")
}

Benchmarking Clamped Vector to Logloss

For a 2-class vector of 1,000,000 observations:

  • Vector A of length=(1000000)
  • Vector B of length=(1000000) with 2 classes
A = [1, 2, 3, 4, ..., 1000000]
B = [0, 1, 1, 0, ...]

Get the following Vector C and D:

C = Clamped A by 1e-15
D = Mean of logloss(C, B)

Initialize data

# How many digits for benchmarking in milliseconds
my_digits <- 6L
# How many runs for benchmarking?
my_runs <- 1000L
# How many observations?
my_obs <- 1000000L
# Generate random data
set.seed(11111)
data <- runif(my_obs, 0, 1)
labels <- round(runif(my_obs, 0, 1), digits = 0)
# Background truth example (no clamping though)
data[1:5]
[1] 0.5014483 0.9702328 0.7876004 0.9022259 0.8141778
labels[1:5]
[1] 0 1 1 0 0
- (labels[1:5] * log(data[1:5]) + (1 - labels[1:5]) * log(1 - data[1:5]))
[1] 0.69604803 0.03021924 0.23876448 2.32509520 1.68296488
mean(- (labels[1:5] * log(data[1:5]) + (1 - labels[1:5]) * log(1 - data[1:5])))
[1] 0.9946184

Benchmarks

# ===== BLOCK 1 =====
faster1 <- function(preds, labels, eps = 1e-15) {
  x <- preds
  x[x < eps] <- eps
  x[x > (1 - eps)] <- 1 - eps
  return(-mean(labels * log(x) + (1 - labels) * log(1 - x)))
}
test_case(faster1, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data1 <- print_well(microbenchmark(faster1(data, labels), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
98.510398 101.366932 102.792063 107.315779 199.486360 111.547776
# ===== BLOCK 2 =====
faster2 <- function(preds, labels, eps = 1e-15) {
  x <- pmin(pmax(preds, eps), 1 - eps)
  return(-mean(labels * log(x) + (1 - labels) * log(1 - x)))
}
test_case(faster2, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data2 <- print_well(microbenchmark(faster2(data, labels), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
98.961620 103.473076 106.699103 118.782302 198.907793 120.656334
# ===== BLOCK 3 =====
faster3 <- function(preds, labels, eps = 1e-15) {
  x <- preds
  x[x < eps] <- eps
  x[x > (1 - eps)] <- 1 - eps
  return(-1/length(labels) * (sum(labels * log(x) + (1 - labels) * log(1 - x))))
}
test_case(faster3, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data3 <- print_well(microbenchmark(faster3(data, labels), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
95.665459 99.602340 101.189218 104.431305 183.179657 105.827194
# ===== BLOCK 4 =====
faster4 <- function(preds, labels, eps = 1e-15) {
  x <- pmin(pmax(preds, eps), 1 - eps)
  return(-1/length(labels) * (sum(labels * log(x) + (1 - labels) * log(1 - x))))
}
test_case(faster4, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data4 <- print_well(microbenchmark(faster4(data, labels), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
98.256468 102.299787 105.832582 121.494859 201.112203 121.317011
# ===== BLOCK 5 =====
cppFunction("double faster5(NumericVector preds, NumericVector labels, double eps) {
  NumericVector clamped = clamp(eps, preds, 1 - eps);
  NumericVector loggy = -1 * ((labels * log(clamped) + (1 - labels) * log(1 - clamped)));
  double logloss = mean(loggy);
  return logloss;
}")
test_case(faster5, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data5 <- print_well(microbenchmark(faster5(data, labels, eps = 1e-15), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
78.355197 79.850368 82.443752 86.616981 150.678771 83.917317
# ===== BLOCK 6 =====
cppFunction("double faster6(NumericVector preds, NumericVector labels, double eps) {
  NumericVector clamped = clamp(eps, preds, 1 - eps);
  NumericVector loggy = -1 * ((labels * log(clamped) + (1 - labels) * log(1 - clamped)));
  double logloss = sum(loggy)/loggy.size();
  return logloss;
}")
test_case(faster6, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data6 <- print_well(microbenchmark(faster6(data, labels, eps = 1e-15), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
77.012176 78.687150 81.050362 85.269969 163.344910 82.639045
# ===== BLOCK 7 =====
faster7 <- function(preds, labels, eps = 1e-15) {
  x <- pmin(pmax(preds, eps), 1 - eps)
  return(-1/length(labels) * sum(log((1 - labels) + (2 * labels - 1) * x)))
}
test_case(faster7, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data7 <- print_well(microbenchmark(faster7(data, labels, eps = 1e-15), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
60.741585 64.389943 66.863298 71.798132 153.880658 78.753778
# ===== BLOCK 8 =====
cppFunction("double faster8(NumericVector preds, NumericVector labels, double eps) {
  int label_size = labels.size();
  NumericVector clamped(label_size);
  clamped = clamp(eps, preds, 1 - eps);
  NumericVector loggy(label_size);
  loggy = -log((1 - labels) + ((2 * labels - 1) * clamped));
  double logloss = sum(loggy) / label_size;
  return logloss;
}")
test_case(faster8, preds = data, labels = labels, eps = 1e-15)

Test case: 0.9837966

data8 <- print_well(microbenchmark(faster8(data, labels, eps = 1e-15), times = my_runs)$time, digits = my_digits)
Min 25% 50% 75% Max Mean
51.169755 51.979825 53.973830 58.375617 124.569496 55.660620

Summary Results

data_time <- data.table(rbindlist(list(data.frame(Time = data1, Bench = "faster1"),
                                       data.frame(Time = data2, Bench = "faster2"),
                                       data.frame(Time = data3, Bench = "faster3"),
                                       data.frame(Time = data4, Bench = "faster4"),
                                       data.frame(Time = data5, Bench = "faster5"),
                                       data.frame(Time = data6, Bench = "faster6"),
                                       data.frame(Time = data7, Bench = "faster7"),
                                       data.frame(Time = data8, Bench = "faster8"))))
data_time <- data_time[, t_mean := mean(Time), by = Bench]
data_time <- data_time[, t_median := median(Time), by = Bench]
data_time$Benchs <- data_time$Bench 
levels(data_time$Benchs) <- paste0("faster", 1:8, "= [", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(min(Time)), by = Bench]$V1), ", ", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(max(Time)), by = Bench]$V1), "], mean=", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(mean(Time)), by = Bench]$V1), ", median=", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(median(Time)), by = Bench]$V1))
my_time <- data_time[, list(min(Time), quantile(Time, probs = 0.25), median(Time), quantile(Time, probs = 0.75), max(Time), mean(Time)), by = Bench]
colnames(my_time) <- c("Function", "Min", "25%", "50%", "75%", "Max", "Mean")
my_time <- my_time[order(Mean, decreasing = FALSE), ]
print(my_time, digits = 6)

Plot Results

ggplotly(ggplot(data = data_time, aes(x = Time)) + geom_histogram(aes(y = ..density..), bins = 20, color = "darkblue", fill = "lightblue") + facet_wrap(~ Benchs, ncol = 2) + geom_vline(aes(xintercept = t_mean), color = "blue", linetype = "dashed", size = 2) + geom_vline(aes(xintercept = t_median), color = "red", linetype = "dashed", size = 2) + labs(x = "Time (Milliseconds)", y = "Density") + theme_bw())
ggplotly(ggplot(data = data_time[, .(Time, Bench)], aes(x = Time, y = ..count.., fill = Bench)) + geom_histogram(aes(y = ..density..), bins = 100, position = "fill") + labs(x = "Time (Milliseconds)", y = "Density") + theme_bw())
ggplotly(ggplot(data = data_time[, .(Time, Bench)], aes(x = Time, y = ..count.., fill = Bench)) + geom_density(position = "fill") + labs(x = "Time (Milliseconds)", y = "Density") + theme_bw())
data_time$MilObs <- (1000 / data_time$Time) * my_obs / 1000000
ggplotly(ggplot(data_time[, .(Bench, MilObs)], aes(x = Bench, y = MilObs, fill = Bench)) + geom_boxplot() + labs(x = "Benchmark", y = "Throughput (Million Obs./s)") + theme_bw())

Scaling Benchmarks

Benchmarker <- function(f, size, runs, digits, name) {
  
  data_runs <- list()
  
  for (i in 1:length(size)) {
    set.seed(11111)
    data <- runif(size[i], 0, 1)
    labels <- round(runif(size[i], 0, 1), digits = 0)
    cat("  \n  \n## ", name, " run: ",format(size[i], big.mark = ",", scientific = FALSE), " samples (", format(runs[i], big.mark = ",", scientific = FALSE), " times)  \n  \n", sep = "")
    test_case(f, preds = data, labels = labels, eps = 1e-15)
    cat("  \n")
    data_runs[[i]] <- print_well(microbenchmark(f(data, labels, eps = 1e-15), times = runs[i])$time, digits = digits)
    data_runs[[i]] <- data.table(Bench = as.factor(paste0("[", i, "] ", format(size[i], big.mark = ",", scientific = FALSE))), Function = as.factor(name), Time = data_runs[[i]])
    gc(verbose = FALSE)
  }
  
  return(data_runs)
  
}
bench_size <- c(100, 1000, 10000, 100000, 1000000, 10000000, 100000000)
bench_runs <- c(10000, 5000, 1000, 500, 100, 50, 10)
run1 <- Benchmarker(faster7, bench_size, bench_runs, my_digits, "Pure R")

Pure R run: 100 samples (10,000 times)

Test case: 1.13218

Min 25% 50% 75% Max Mean
0.020527 0.022808 0.023569 0.034972 4.309224 0.029422

Pure R run: 1,000 samples (5,000 times)

Test case: 1.193473

Min 25% 50% 75% Max Mean
0.061202 0.068805 0.069945 0.076407 3.311367 0.077917

Pure R run: 10,000 samples (1,000 times)

Test case: 1.159039

Min 25% 50% 75% Max Mean
0.481632 0.532571 0.537513 0.618862 4.236619 0.605984

Pure R run: 100,000 samples (500 times)

Test case: 0.9539782

Min 25% 50% 75% Max Mean
4.980164 5.626111 5.798597 6.117342 84.356026 6.520741

Pure R run: 1,000,000 samples (100 times)

Test case: 0.9837966

Min 25% 50% 75% Max Mean
60.215477 64.248722 65.580910 67.840913 151.788769 73.396825

Pure R run: 10,000,000 samples (50 times)

Test case: 0.9312716

Min 25% 50% 75% Max Mean
604.348530 632.697758 649.104247 701.033229 776.059097 665.486103

Pure R run: 100,000,000 samples (10 times)

Test case: 1.413474

Min 25% 50% 75% Max Mean
6321.118923 6364.084769 6535.476782 6613.934425 6681.649989 6503.505692
run2 <- Benchmarker(faster8, bench_size, bench_runs, my_digits, "Rcpp")

Rcpp run: 100 samples (10,000 times)

Test case: 1.13218

Min 25% 50% 75% Max Mean
0.005702 0.006082 0.006462 0.006843 0.107199 0.007003

Rcpp run: 1,000 samples (5,000 times)

Test case: 1.193473

Min 25% 50% 75% Max Mean
0.045996 0.049418 0.050938 0.052079 0.137229 0.052078

Rcpp run: 10,000 samples (1,000 times)

Test case: 1.159039

Min 25% 50% 75% Max Mean
0.460725 0.486575 0.511853 0.555664 2.341640 0.547915

Rcpp run: 100,000 samples (500 times)

Test case: 0.9539782

Min 25% 50% 75% Max Mean
4.611053 5.099052 5.235616 5.437088 8.441685 5.341539

Rcpp run: 1,000,000 samples (100 times)

Test case: 0.9837966

Min 25% 50% 75% Max Mean
47.781602 51.192848 51.665453 52.588518 118.033053 53.538399

Rcpp run: 10,000,000 samples (50 times)

Test case: 0.9312716

Min 25% 50% 75% Max Mean
515.161353 528.701044 554.947169 566.630370 628.325622 549.245088

Rcpp run: 100,000,000 samples (10 times)

Test case: 1.413474

Min 25% 50% 75% Max Mean
5351.091904 5365.159034 5511.260205 5548.570193 5586.359344 5469.415267

Scaling Results

run1_all <- rbindlist(run1)
run2_all <- rbindlist(run2)
run_all <- rbind(run1_all, run2_all)
run_all$Repeats <- rep(inverse.rle(list(lengths = bench_runs, values = bench_size)), 2)
run_all$MilObs <- (1000 / run_all$Time) * run_all$Repeats / 1000000
run_time <- run_all[, list(quantile(Time, probs = 0.05), median(Time), quantile(Time, probs = 0.95), mean(Time)), by = list(Function, Bench)]
colnames(run_time) <- c("Function", "Benchmark", "5%", "50%", "95%", "Mean")
run_time$`Mil.Obs/s` <- (1000 / run_time$Mean) * bench_size / 1000000
run_time$`5%` <- format(run_time$`5%`, digits = 6, scientific = FALSE)
run_time$`50%` <- format(run_time$`50%`, digits = 6, scientific = FALSE)
run_time$`95%` <- format(run_time$`95%`, digits = 6, scientific = FALSE)
run_time$Mean <- format(run_time$Mean, digits = 6, scientific = FALSE)
run_time$`Mil.Obs/s` <- format(run_time$`Mil.Obs/s`, digits = 6, scientific = FALSE)
print(run_time[1:(nrow(run_time) / 2)])
print(run_time[(nrow(run_time) / 2 + 1):nrow(run_time)])
ggplot(run_all, aes(x = Bench, y = Time, color = Function, fill = Bench)) + geom_boxplot() + scale_y_log10(labels = scales::comma, breaks = c(0.01, 0.1, 1, 10, 100, 1000, 10000)) + stat_summary(fun.y = mean, geom = "line", aes(group = Function)) + stat_summary(fun.y = mean, geom = "point", aes(group = Function)) + labs(x = "Benchmark", y = "Time (Milliseconds)") + theme_bw()

ggplot(run_all, aes(x = Bench, y = MilObs, color = Function, fill = Bench)) + geom_boxplot() + scale_y_log10(labels = scales::comma, breaks = c(1, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25), limits = c(1, NA)) + stat_summary(fun.y = mean, geom = "line", aes(group = Function)) + stat_summary(fun.y = mean, geom = "point", aes(group = Function)) + labs(x = "Benchmark", y = "Throughput (Million Obs./s)") + theme_bw()

ggplot(data = run_all, aes(x = MilObs, color = Function, fill = Function, group = Function)) + coord_flip() + stat_ecdf(aes(ymin = ..y.., ymax = 1), alpha = 0.5, geom = "ribbon") + stat_ecdf(geom = "line", size = 2, alpha = 0.75, pad = FALSE) + labs(x = "Throughput (Million Obs./s)", y = "Percentile") + facet_wrap(~ Bench, dir = "h", ncol = 2, scales = "free") + theme_bw()

LS0tDQp0aXRsZTogIkJlbmNobWFya3M6IExvZ2xvc3MiDQpvdXRwdXQ6DQogIGh0bWxfbm90ZWJvb2s6DQogICAgY29sbGFwc2VkOiBubw0KICAgIHRoZW1lOiB1bml0ZWQNCiAgICB0b2M6IHllcw0KICAgIHRvY19kZXB0aDogMQ0KICAgIHRvY19mbG9hdDogeWVzDQotLS0NCg0KVGhpcyBub3RlYm9vayBpcyBhbGwgYWJvdXQgYmVuY2htYXJraW5nIHNvbWUgUiBjb2RlIHVzZWQgaW4gdGhpcyBwYWNrYWdlLg0KDQpIYXJkd2FyZSAvIFNvZnR3YXJlIHVzZWQ6DQoNCiogSW50ZWwgaTctNDYwMFUNCiogQ29tcGlsYXRpb24gZmxhZ3MgZm9yIEMvQysrOiBgLU8yIC1XYWxsICQoREVCVUdGTEFHKSAtbXR1bmU9Y29yZTJgIChSJ3MgZGVmYXVsdHMpDQoqIFdpbmRvd3MgU2VydmVyIDIwMTIgUjINCiogUiAzLjMuMiArIEludGVsIE1LTA0KDQojIExpYnJhcmllcw0KDQpgYGB7ciBpbml0fQ0KbGlicmFyeShkYXRhLnRhYmxlKQ0KbGlicmFyeShtaWNyb2JlbmNobWFyaykNCmxpYnJhcnkoUmNwcCkNCmxpYnJhcnkoZ2dwbG90MikNCmxpYnJhcnkocGxvdGx5KQ0KYGBgDQoNCmBgYHtyIGJhc2VkfQ0KDQojIEhlbHBlciBmdW5jdGlvbiB0byBwcmludCBkYXRhIHdlbGwgaW4gdGFibGVzDQpwcmludF93ZWxsIDwtIGZ1bmN0aW9uKGRhdGEsIGRpZ2l0cyA9IDYpIHsNCiAgDQogICMgVG8gbWlsbGlzZWNvbmRzDQogIGRhdGEgPC0gZGF0YSAvIDEwMDAwMDANCiAgDQogICMgU3ByaW50ZiBoZWxwZXINCiAgc3ByaW50Zl9oZWxwZXIgPC0gcGFzdGUwKCIlLjAiLCBkaWdpdHMsICJmIikNCiAgDQogIGNhdCgifCBNaW4gfCAyNSUgfCA1MCUgfCA3NSUgfCBNYXggfCBNZWFuIHwgIFxufCAtLTogfCAtLTogfCAtLTogfCAtLTogfCAtLTogfCAtLTogfCAgXG58ICIsIHNwcmludGYoc3ByaW50Zl9oZWxwZXIsIG1pbihkYXRhKSksICIgfCAiLCBzcHJpbnRmKHNwcmludGZfaGVscGVyLCBxdWFudGlsZShkYXRhLCBwcm9icyA9IDAuMjUpKSwgIiB8ICIsIHNwcmludGYoc3ByaW50Zl9oZWxwZXIsIG1lZGlhbihkYXRhKSksICIgfCAiLCBzcHJpbnRmKHNwcmludGZfaGVscGVyLCBxdWFudGlsZShkYXRhLCBwcm9icyA9IDAuNzUpKSwgIiB8ICIsIHNwcmludGYoc3ByaW50Zl9oZWxwZXIsIG1heChkYXRhKSksICIgfCAiLCBzcHJpbnRmKHNwcmludGZfaGVscGVyLCBtZWFuKGRhdGEpKSwgIiB8ICBcbiIsIHNlcCA9ICIiKQ0KICANCiAgcmV0dXJuKGRhdGEpDQogIA0KfQ0KDQojIFRlc3QgY2FzZSBmdW5jdGlvbg0KIyBBcmd1bWVudHMgcmVuYW1lZCB0byBhdm9pZCByZWN1cnNpdmUgY2xhc2gNCnRlc3RfY2FzZSA8LSBmdW5jdGlvbihmLCBwcmVkcywgbGFiZWxzLCBlcHMpIHsNCiAgY2F0KCJUZXN0IGNhc2U6ICIsIGRvLmNhbGwoZiwgbGlzdChwcmVkcyA9IHByZWRzWzE6NTBdLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxhYmVscyA9IGxhYmVsc1sxOjUwXSwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlcHMgPSAxZS0xNSkpLCAiICBcbiIsIHNlcCA9ICIiKQ0KfQ0KDQpgYGANCg0KIyBCZW5jaG1hcmtpbmcgQ2xhbXBlZCBWZWN0b3IgdG8gTG9nbG9zcw0KDQpGb3IgYSAyLWNsYXNzIHZlY3RvciBvZiAxLDAwMCwwMDAgb2JzZXJ2YXRpb25zOg0KDQoqIFZlY3RvciBBIG9mIGxlbmd0aD0oMTAwMDAwMCkNCiogVmVjdG9yIEIgb2YgbGVuZ3RoPSgxMDAwMDAwKSB3aXRoIDIgY2xhc3Nlcw0KDQpgYGANCkEgPSBbMSwgMiwgMywgNCwgLi4uLCAxMDAwMDAwXQ0KQiA9IFswLCAxLCAxLCAwLCAuLi5dDQpgYGANCg0KR2V0IHRoZSBmb2xsb3dpbmcgVmVjdG9yIEMgYW5kIEQ6DQoNCmBgYA0KQyA9IENsYW1wZWQgQSBieSAxZS0xNQ0KRCA9IE1lYW4gb2YgbG9nbG9zcyhDLCBCKQ0KYGBgDQoNCiMgSW5pdGlhbGl6ZSBkYXRhDQoNCmBgYHtyIGJlbmNoMX0NCg0KIyBIb3cgbWFueSBkaWdpdHMgZm9yIGJlbmNobWFya2luZyBpbiBtaWxsaXNlY29uZHMNCm15X2RpZ2l0cyA8LSA2TA0KDQojIEhvdyBtYW55IHJ1bnMgZm9yIGJlbmNobWFya2luZz8NCm15X3J1bnMgPC0gMTAwMEwNCg0KIyBIb3cgbWFueSBvYnNlcnZhdGlvbnM/DQpteV9vYnMgPC0gMTAwMDAwMEwNCg0KIyBHZW5lcmF0ZSByYW5kb20gZGF0YQ0Kc2V0LnNlZWQoMTExMTEpDQpkYXRhIDwtIHJ1bmlmKG15X29icywgMCwgMSkNCmxhYmVscyA8LSByb3VuZChydW5pZihteV9vYnMsIDAsIDEpLCBkaWdpdHMgPSAwKQ0KDQojIEJhY2tncm91bmQgdHJ1dGggZXhhbXBsZSAobm8gY2xhbXBpbmcgdGhvdWdoKQ0KZGF0YVsxOjVdDQpsYWJlbHNbMTo1XQ0KLSAobGFiZWxzWzE6NV0gKiBsb2coZGF0YVsxOjVdKSArICgxIC0gbGFiZWxzWzE6NV0pICogbG9nKDEgLSBkYXRhWzE6NV0pKQ0KbWVhbigtIChsYWJlbHNbMTo1XSAqIGxvZyhkYXRhWzE6NV0pICsgKDEgLSBsYWJlbHNbMTo1XSkgKiBsb2coMSAtIGRhdGFbMTo1XSkpKQ0KDQpgYGANCg0KIyBCZW5jaG1hcmtzDQoNCmBgYHtyIGJlbmNoMiwgcmVzdWx0cz0iYXNpcyJ9DQoNCiMgPT09PT0gQkxPQ0sgMSA9PT09PQ0KZmFzdGVyMSA8LSBmdW5jdGlvbihwcmVkcywgbGFiZWxzLCBlcHMgPSAxZS0xNSkgew0KICB4IDwtIHByZWRzDQogIHhbeCA8IGVwc10gPC0gZXBzDQogIHhbeCA+ICgxIC0gZXBzKV0gPC0gMSAtIGVwcw0KICByZXR1cm4oLW1lYW4obGFiZWxzICogbG9nKHgpICsgKDEgLSBsYWJlbHMpICogbG9nKDEgLSB4KSkpDQp9DQp0ZXN0X2Nhc2UoZmFzdGVyMSwgcHJlZHMgPSBkYXRhLCBsYWJlbHMgPSBsYWJlbHMsIGVwcyA9IDFlLTE1KQ0KZGF0YTEgPC0gcHJpbnRfd2VsbChtaWNyb2JlbmNobWFyayhmYXN0ZXIxKGRhdGEsIGxhYmVscyksIHRpbWVzID0gbXlfcnVucykkdGltZSwgZGlnaXRzID0gbXlfZGlnaXRzKQ0KDQojID09PT09IEJMT0NLIDIgPT09PT0NCmZhc3RlcjIgPC0gZnVuY3Rpb24ocHJlZHMsIGxhYmVscywgZXBzID0gMWUtMTUpIHsNCiAgeCA8LSBwbWluKHBtYXgocHJlZHMsIGVwcyksIDEgLSBlcHMpDQogIHJldHVybigtbWVhbihsYWJlbHMgKiBsb2coeCkgKyAoMSAtIGxhYmVscykgKiBsb2coMSAtIHgpKSkNCn0NCnRlc3RfY2FzZShmYXN0ZXIyLCBwcmVkcyA9IGRhdGEsIGxhYmVscyA9IGxhYmVscywgZXBzID0gMWUtMTUpDQpkYXRhMiA8LSBwcmludF93ZWxsKG1pY3JvYmVuY2htYXJrKGZhc3RlcjIoZGF0YSwgbGFiZWxzKSwgdGltZXMgPSBteV9ydW5zKSR0aW1lLCBkaWdpdHMgPSBteV9kaWdpdHMpDQoNCiMgPT09PT0gQkxPQ0sgMyA9PT09PQ0KZmFzdGVyMyA8LSBmdW5jdGlvbihwcmVkcywgbGFiZWxzLCBlcHMgPSAxZS0xNSkgew0KICB4IDwtIHByZWRzDQogIHhbeCA8IGVwc10gPC0gZXBzDQogIHhbeCA+ICgxIC0gZXBzKV0gPC0gMSAtIGVwcw0KICByZXR1cm4oLTEvbGVuZ3RoKGxhYmVscykgKiAoc3VtKGxhYmVscyAqIGxvZyh4KSArICgxIC0gbGFiZWxzKSAqIGxvZygxIC0geCkpKSkNCn0NCnRlc3RfY2FzZShmYXN0ZXIzLCBwcmVkcyA9IGRhdGEsIGxhYmVscyA9IGxhYmVscywgZXBzID0gMWUtMTUpDQpkYXRhMyA8LSBwcmludF93ZWxsKG1pY3JvYmVuY2htYXJrKGZhc3RlcjMoZGF0YSwgbGFiZWxzKSwgdGltZXMgPSBteV9ydW5zKSR0aW1lLCBkaWdpdHMgPSBteV9kaWdpdHMpDQoNCiMgPT09PT0gQkxPQ0sgNCA9PT09PQ0KZmFzdGVyNCA8LSBmdW5jdGlvbihwcmVkcywgbGFiZWxzLCBlcHMgPSAxZS0xNSkgew0KICB4IDwtIHBtaW4ocG1heChwcmVkcywgZXBzKSwgMSAtIGVwcykNCiAgcmV0dXJuKC0xL2xlbmd0aChsYWJlbHMpICogKHN1bShsYWJlbHMgKiBsb2coeCkgKyAoMSAtIGxhYmVscykgKiBsb2coMSAtIHgpKSkpDQp9DQp0ZXN0X2Nhc2UoZmFzdGVyNCwgcHJlZHMgPSBkYXRhLCBsYWJlbHMgPSBsYWJlbHMsIGVwcyA9IDFlLTE1KQ0KZGF0YTQgPC0gcHJpbnRfd2VsbChtaWNyb2JlbmNobWFyayhmYXN0ZXI0KGRhdGEsIGxhYmVscyksIHRpbWVzID0gbXlfcnVucykkdGltZSwgZGlnaXRzID0gbXlfZGlnaXRzKQ0KDQojID09PT09IEJMT0NLIDUgPT09PT0NCmNwcEZ1bmN0aW9uKCJkb3VibGUgZmFzdGVyNShOdW1lcmljVmVjdG9yIHByZWRzLCBOdW1lcmljVmVjdG9yIGxhYmVscywgZG91YmxlIGVwcykgew0KICBOdW1lcmljVmVjdG9yIGNsYW1wZWQgPSBjbGFtcChlcHMsIHByZWRzLCAxIC0gZXBzKTsNCiAgTnVtZXJpY1ZlY3RvciBsb2dneSA9IC0xICogKChsYWJlbHMgKiBsb2coY2xhbXBlZCkgKyAoMSAtIGxhYmVscykgKiBsb2coMSAtIGNsYW1wZWQpKSk7DQogIGRvdWJsZSBsb2dsb3NzID0gbWVhbihsb2dneSk7DQogIHJldHVybiBsb2dsb3NzOw0KfSIpDQp0ZXN0X2Nhc2UoZmFzdGVyNSwgcHJlZHMgPSBkYXRhLCBsYWJlbHMgPSBsYWJlbHMsIGVwcyA9IDFlLTE1KQ0KZGF0YTUgPC0gcHJpbnRfd2VsbChtaWNyb2JlbmNobWFyayhmYXN0ZXI1KGRhdGEsIGxhYmVscywgZXBzID0gMWUtMTUpLCB0aW1lcyA9IG15X3J1bnMpJHRpbWUsIGRpZ2l0cyA9IG15X2RpZ2l0cykNCg0KIyA9PT09PSBCTE9DSyA2ID09PT09DQpjcHBGdW5jdGlvbigiZG91YmxlIGZhc3RlcjYoTnVtZXJpY1ZlY3RvciBwcmVkcywgTnVtZXJpY1ZlY3RvciBsYWJlbHMsIGRvdWJsZSBlcHMpIHsNCiAgTnVtZXJpY1ZlY3RvciBjbGFtcGVkID0gY2xhbXAoZXBzLCBwcmVkcywgMSAtIGVwcyk7DQogIE51bWVyaWNWZWN0b3IgbG9nZ3kgPSAtMSAqICgobGFiZWxzICogbG9nKGNsYW1wZWQpICsgKDEgLSBsYWJlbHMpICogbG9nKDEgLSBjbGFtcGVkKSkpOw0KICBkb3VibGUgbG9nbG9zcyA9IHN1bShsb2dneSkvbG9nZ3kuc2l6ZSgpOw0KICByZXR1cm4gbG9nbG9zczsNCn0iKQ0KdGVzdF9jYXNlKGZhc3RlcjYsIHByZWRzID0gZGF0YSwgbGFiZWxzID0gbGFiZWxzLCBlcHMgPSAxZS0xNSkNCmRhdGE2IDwtIHByaW50X3dlbGwobWljcm9iZW5jaG1hcmsoZmFzdGVyNihkYXRhLCBsYWJlbHMsIGVwcyA9IDFlLTE1KSwgdGltZXMgPSBteV9ydW5zKSR0aW1lLCBkaWdpdHMgPSBteV9kaWdpdHMpDQoNCiMgPT09PT0gQkxPQ0sgNyA9PT09PQ0KZmFzdGVyNyA8LSBmdW5jdGlvbihwcmVkcywgbGFiZWxzLCBlcHMgPSAxZS0xNSkgew0KICB4IDwtIHBtaW4ocG1heChwcmVkcywgZXBzKSwgMSAtIGVwcykNCiAgcmV0dXJuKC0xL2xlbmd0aChsYWJlbHMpICogc3VtKGxvZygoMSAtIGxhYmVscykgKyAoMiAqIGxhYmVscyAtIDEpICogeCkpKQ0KfQ0KdGVzdF9jYXNlKGZhc3RlcjcsIHByZWRzID0gZGF0YSwgbGFiZWxzID0gbGFiZWxzLCBlcHMgPSAxZS0xNSkNCmRhdGE3IDwtIHByaW50X3dlbGwobWljcm9iZW5jaG1hcmsoZmFzdGVyNyhkYXRhLCBsYWJlbHMsIGVwcyA9IDFlLTE1KSwgdGltZXMgPSBteV9ydW5zKSR0aW1lLCBkaWdpdHMgPSBteV9kaWdpdHMpDQoNCiMgPT09PT0gQkxPQ0sgOCA9PT09PQ0KY3BwRnVuY3Rpb24oImRvdWJsZSBmYXN0ZXI4KE51bWVyaWNWZWN0b3IgcHJlZHMsIE51bWVyaWNWZWN0b3IgbGFiZWxzLCBkb3VibGUgZXBzKSB7DQogIGludCBsYWJlbF9zaXplID0gbGFiZWxzLnNpemUoKTsNCiAgTnVtZXJpY1ZlY3RvciBjbGFtcGVkKGxhYmVsX3NpemUpOw0KICBjbGFtcGVkID0gY2xhbXAoZXBzLCBwcmVkcywgMSAtIGVwcyk7DQogIE51bWVyaWNWZWN0b3IgbG9nZ3kobGFiZWxfc2l6ZSk7DQogIGxvZ2d5ID0gLWxvZygoMSAtIGxhYmVscykgKyAoKDIgKiBsYWJlbHMgLSAxKSAqIGNsYW1wZWQpKTsNCiAgZG91YmxlIGxvZ2xvc3MgPSBzdW0obG9nZ3kpIC8gbGFiZWxfc2l6ZTsNCiAgcmV0dXJuIGxvZ2xvc3M7DQp9IikNCnRlc3RfY2FzZShmYXN0ZXI4LCBwcmVkcyA9IGRhdGEsIGxhYmVscyA9IGxhYmVscywgZXBzID0gMWUtMTUpDQpkYXRhOCA8LSBwcmludF93ZWxsKG1pY3JvYmVuY2htYXJrKGZhc3RlcjgoZGF0YSwgbGFiZWxzLCBlcHMgPSAxZS0xNSksIHRpbWVzID0gbXlfcnVucykkdGltZSwgZGlnaXRzID0gbXlfZGlnaXRzKQ0KDQpgYGANCg0KIyBTdW1tYXJ5IFJlc3VsdHMNCg0KYGBge3IgYmVuY2gzfQ0KDQpkYXRhX3RpbWUgPC0gZGF0YS50YWJsZShyYmluZGxpc3QobGlzdChkYXRhLmZyYW1lKFRpbWUgPSBkYXRhMSwgQmVuY2ggPSAiZmFzdGVyMSIpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YS5mcmFtZShUaW1lID0gZGF0YTIsIEJlbmNoID0gImZhc3RlcjIiKSwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEuZnJhbWUoVGltZSA9IGRhdGEzLCBCZW5jaCA9ICJmYXN0ZXIzIiksDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhLmZyYW1lKFRpbWUgPSBkYXRhNCwgQmVuY2ggPSAiZmFzdGVyNCIpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YS5mcmFtZShUaW1lID0gZGF0YTUsIEJlbmNoID0gImZhc3RlcjUiKSwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEuZnJhbWUoVGltZSA9IGRhdGE2LCBCZW5jaCA9ICJmYXN0ZXI2IiksDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhLmZyYW1lKFRpbWUgPSBkYXRhNywgQmVuY2ggPSAiZmFzdGVyNyIpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YS5mcmFtZShUaW1lID0gZGF0YTgsIEJlbmNoID0gImZhc3RlcjgiKSkpKQ0KZGF0YV90aW1lIDwtIGRhdGFfdGltZVssIHRfbWVhbiA6PSBtZWFuKFRpbWUpLCBieSA9IEJlbmNoXQ0KZGF0YV90aW1lIDwtIGRhdGFfdGltZVssIHRfbWVkaWFuIDo9IG1lZGlhbihUaW1lKSwgYnkgPSBCZW5jaF0NCmRhdGFfdGltZSRCZW5jaHMgPC0gZGF0YV90aW1lJEJlbmNoIA0KbGV2ZWxzKGRhdGFfdGltZSRCZW5jaHMpIDwtIHBhc3RlMCgiZmFzdGVyIiwgMTo4LCAiPSBbIiwgc3ByaW50ZihwYXN0ZTAoIiUuMCIsIG15X2RpZ2l0cywgImYiKSwgZGF0YV90aW1lWywgbGlzdChtaW4oVGltZSkpLCBieSA9IEJlbmNoXSRWMSksICIsICIsIHNwcmludGYocGFzdGUwKCIlLjAiLCBteV9kaWdpdHMsICJmIiksIGRhdGFfdGltZVssIGxpc3QobWF4KFRpbWUpKSwgYnkgPSBCZW5jaF0kVjEpLCAiXSwgbWVhbj0iLCBzcHJpbnRmKHBhc3RlMCgiJS4wIiwgbXlfZGlnaXRzLCAiZiIpLCBkYXRhX3RpbWVbLCBsaXN0KG1lYW4oVGltZSkpLCBieSA9IEJlbmNoXSRWMSksICIsIG1lZGlhbj0iLCBzcHJpbnRmKHBhc3RlMCgiJS4wIiwgbXlfZGlnaXRzLCAiZiIpLCBkYXRhX3RpbWVbLCBsaXN0KG1lZGlhbihUaW1lKSksIGJ5ID0gQmVuY2hdJFYxKSkNCg0KbXlfdGltZSA8LSBkYXRhX3RpbWVbLCBsaXN0KG1pbihUaW1lKSwgcXVhbnRpbGUoVGltZSwgcHJvYnMgPSAwLjI1KSwgbWVkaWFuKFRpbWUpLCBxdWFudGlsZShUaW1lLCBwcm9icyA9IDAuNzUpLCBtYXgoVGltZSksIG1lYW4oVGltZSkpLCBieSA9IEJlbmNoXQ0KY29sbmFtZXMobXlfdGltZSkgPC0gYygiRnVuY3Rpb24iLCAiTWluIiwgIjI1JSIsICI1MCUiLCAiNzUlIiwgIk1heCIsICJNZWFuIikNCm15X3RpbWUgPC0gbXlfdGltZVtvcmRlcihNZWFuLCBkZWNyZWFzaW5nID0gRkFMU0UpLCBdDQpwcmludChteV90aW1lLCBkaWdpdHMgPSA2KQ0KDQpgYGANCg0KIyBQbG90IFJlc3VsdHMNCg0KYGBge3IgYmVuY2g0LCBmaWcuaGVpZ2h0PTksIGZpZy53aWR0aD0xMH0NCg0KZ2dwbG90bHkoZ2dwbG90KGRhdGEgPSBkYXRhX3RpbWUsIGFlcyh4ID0gVGltZSkpICsgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSAuLmRlbnNpdHkuLiksIGJpbnMgPSAyMCwgY29sb3IgPSAiZGFya2JsdWUiLCBmaWxsID0gImxpZ2h0Ymx1ZSIpICsgZmFjZXRfd3JhcCh+IEJlbmNocywgbmNvbCA9IDIpICsgZ2VvbV92bGluZShhZXMoeGludGVyY2VwdCA9IHRfbWVhbiksIGNvbG9yID0gImJsdWUiLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBzaXplID0gMikgKyBnZW9tX3ZsaW5lKGFlcyh4aW50ZXJjZXB0ID0gdF9tZWRpYW4pLCBjb2xvciA9ICJyZWQiLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBzaXplID0gMikgKyBsYWJzKHggPSAiVGltZSAoTWlsbGlzZWNvbmRzKSIsIHkgPSAiRGVuc2l0eSIpICsgdGhlbWVfYncoKSkNCg0KYGBgDQoNCmBgYHtyIGJlbmNoNSwgZmlnLmhlaWdodD02LCBmaWcud2lkdGg9MTB9DQpnZ3Bsb3RseShnZ3Bsb3QoZGF0YSA9IGRhdGFfdGltZVssIC4oVGltZSwgQmVuY2gpXSwgYWVzKHggPSBUaW1lLCB5ID0gLi5jb3VudC4uLCBmaWxsID0gQmVuY2gpKSArIGdlb21faGlzdG9ncmFtKGFlcyh5ID0gLi5kZW5zaXR5Li4pLCBiaW5zID0gMTAwLCBwb3NpdGlvbiA9ICJmaWxsIikgKyBsYWJzKHggPSAiVGltZSAoTWlsbGlzZWNvbmRzKSIsIHkgPSAiRGVuc2l0eSIpICsgdGhlbWVfYncoKSkNCmBgYA0KDQpgYGB7ciBiZW5jaDYsIGZpZy5oZWlnaHQ9NiwgZmlnLndpZHRoPTEwfQ0KZ2dwbG90bHkoZ2dwbG90KGRhdGEgPSBkYXRhX3RpbWVbLCAuKFRpbWUsIEJlbmNoKV0sIGFlcyh4ID0gVGltZSwgeSA9IC4uY291bnQuLiwgZmlsbCA9IEJlbmNoKSkgKyBnZW9tX2RlbnNpdHkocG9zaXRpb24gPSAiZmlsbCIpICsgbGFicyh4ID0gIlRpbWUgKE1pbGxpc2Vjb25kcykiLCB5ID0gIkRlbnNpdHkiKSArIHRoZW1lX2J3KCkpDQpgYGANCg0KYGBge3IgYmVuY2g3LCBmaWcuaGVpZ2h0PTYsIGZpZy53aWR0aD0xMH0NCg0KZGF0YV90aW1lJE1pbE9icyA8LSAoMTAwMCAvIGRhdGFfdGltZSRUaW1lKSAqIG15X29icyAvIDEwMDAwMDANCmdncGxvdGx5KGdncGxvdChkYXRhX3RpbWVbLCAuKEJlbmNoLCBNaWxPYnMpXSwgYWVzKHggPSBCZW5jaCwgeSA9IE1pbE9icywgZmlsbCA9IEJlbmNoKSkgKyBnZW9tX2JveHBsb3QoKSArIGxhYnMoeCA9ICJCZW5jaG1hcmsiLCB5ID0gIlRocm91Z2hwdXQgKE1pbGxpb24gT2JzLi9zKSIpICsgdGhlbWVfYncoKSkNCg0KYGBgDQoNCiMgU2NhbGluZyBCZW5jaG1hcmtzDQoNCmBgYHtyIGJlbmNoX3NjYWxlMX0NCg0KQmVuY2htYXJrZXIgPC0gZnVuY3Rpb24oZiwgc2l6ZSwgcnVucywgZGlnaXRzLCBuYW1lKSB7DQogIA0KICBkYXRhX3J1bnMgPC0gbGlzdCgpDQogIA0KICBmb3IgKGkgaW4gMTpsZW5ndGgoc2l6ZSkpIHsNCiAgICBzZXQuc2VlZCgxMTExMSkNCiAgICBkYXRhIDwtIHJ1bmlmKHNpemVbaV0sIDAsIDEpDQogICAgbGFiZWxzIDwtIHJvdW5kKHJ1bmlmKHNpemVbaV0sIDAsIDEpLCBkaWdpdHMgPSAwKQ0KICAgIGNhdCgiICBcbiAgXG4jIyAiLCBuYW1lLCAiIHJ1bjogIixmb3JtYXQoc2l6ZVtpXSwgYmlnLm1hcmsgPSAiLCIsIHNjaWVudGlmaWMgPSBGQUxTRSksICIgc2FtcGxlcyAoIiwgZm9ybWF0KHJ1bnNbaV0sIGJpZy5tYXJrID0gIiwiLCBzY2llbnRpZmljID0gRkFMU0UpLCAiIHRpbWVzKSAgXG4gIFxuIiwgc2VwID0gIiIpDQogICAgdGVzdF9jYXNlKGYsIHByZWRzID0gZGF0YSwgbGFiZWxzID0gbGFiZWxzLCBlcHMgPSAxZS0xNSkNCiAgICBjYXQoIiAgXG4iKQ0KICAgIGRhdGFfcnVuc1tbaV1dIDwtIHByaW50X3dlbGwobWljcm9iZW5jaG1hcmsoZihkYXRhLCBsYWJlbHMsIGVwcyA9IDFlLTE1KSwgdGltZXMgPSBydW5zW2ldKSR0aW1lLCBkaWdpdHMgPSBkaWdpdHMpDQogICAgZGF0YV9ydW5zW1tpXV0gPC0gZGF0YS50YWJsZShCZW5jaCA9IGFzLmZhY3RvcihwYXN0ZTAoIlsiLCBpLCAiXSAiLCBmb3JtYXQoc2l6ZVtpXSwgYmlnLm1hcmsgPSAiLCIsIHNjaWVudGlmaWMgPSBGQUxTRSkpKSwgRnVuY3Rpb24gPSBhcy5mYWN0b3IobmFtZSksIFRpbWUgPSBkYXRhX3J1bnNbW2ldXSkNCiAgICBnYyh2ZXJib3NlID0gRkFMU0UpDQogIH0NCiAgDQogIHJldHVybihkYXRhX3J1bnMpDQogIA0KfQ0KDQpiZW5jaF9zaXplIDwtIGMoMTAwLCAxMDAwLCAxMDAwMCwgMTAwMDAwLCAxMDAwMDAwLCAxMDAwMDAwMCwgMTAwMDAwMDAwKQ0KYmVuY2hfcnVucyA8LSBjKDEwMDAwLCA1MDAwLCAxMDAwLCA1MDAsIDEwMCwgNTAsIDEwKQ0KDQpgYGANCg0KYGBge3IgYmVuY2hfc2NhbGUyLCByZXN1bHRzPSJhc2lzIn0NCg0KcnVuMSA8LSBCZW5jaG1hcmtlcihmYXN0ZXI3LCBiZW5jaF9zaXplLCBiZW5jaF9ydW5zLCBteV9kaWdpdHMsICJQdXJlIFIiKQ0KcnVuMiA8LSBCZW5jaG1hcmtlcihmYXN0ZXI4LCBiZW5jaF9zaXplLCBiZW5jaF9ydW5zLCBteV9kaWdpdHMsICJSY3BwIikNCg0KYGBgDQoNCiMgU2NhbGluZyBSZXN1bHRzDQoNCmBgYHtyIGJlbmNoX3NjYWxlM30NCg0KcnVuMV9hbGwgPC0gcmJpbmRsaXN0KHJ1bjEpDQpydW4yX2FsbCA8LSByYmluZGxpc3QocnVuMikNCnJ1bl9hbGwgPC0gcmJpbmQocnVuMV9hbGwsIHJ1bjJfYWxsKQ0KcnVuX2FsbCRSZXBlYXRzIDwtIHJlcChpbnZlcnNlLnJsZShsaXN0KGxlbmd0aHMgPSBiZW5jaF9ydW5zLCB2YWx1ZXMgPSBiZW5jaF9zaXplKSksIDIpDQpydW5fYWxsJE1pbE9icyA8LSAoMTAwMCAvIHJ1bl9hbGwkVGltZSkgKiBydW5fYWxsJFJlcGVhdHMgLyAxMDAwMDAwDQpydW5fdGltZSA8LSBydW5fYWxsWywgbGlzdChxdWFudGlsZShUaW1lLCBwcm9icyA9IDAuMDUpLCBtZWRpYW4oVGltZSksIHF1YW50aWxlKFRpbWUsIHByb2JzID0gMC45NSksIG1lYW4oVGltZSkpLCBieSA9IGxpc3QoRnVuY3Rpb24sIEJlbmNoKV0NCmNvbG5hbWVzKHJ1bl90aW1lKSA8LSBjKCJGdW5jdGlvbiIsICJCZW5jaG1hcmsiLCAiNSUiLCAiNTAlIiwgIjk1JSIsICJNZWFuIikNCnJ1bl90aW1lJGBNaWwuT2JzL3NgIDwtICgxMDAwIC8gcnVuX3RpbWUkTWVhbikgKiBiZW5jaF9zaXplIC8gMTAwMDAwMA0KcnVuX3RpbWUkYDUlYCA8LSBmb3JtYXQocnVuX3RpbWUkYDUlYCwgZGlnaXRzID0gNiwgc2NpZW50aWZpYyA9IEZBTFNFKQ0KcnVuX3RpbWUkYDUwJWAgPC0gZm9ybWF0KHJ1bl90aW1lJGA1MCVgLCBkaWdpdHMgPSA2LCBzY2llbnRpZmljID0gRkFMU0UpDQpydW5fdGltZSRgOTUlYCA8LSBmb3JtYXQocnVuX3RpbWUkYDk1JWAsIGRpZ2l0cyA9IDYsIHNjaWVudGlmaWMgPSBGQUxTRSkNCnJ1bl90aW1lJE1lYW4gPC0gZm9ybWF0KHJ1bl90aW1lJE1lYW4sIGRpZ2l0cyA9IDYsIHNjaWVudGlmaWMgPSBGQUxTRSkNCnJ1bl90aW1lJGBNaWwuT2JzL3NgIDwtIGZvcm1hdChydW5fdGltZSRgTWlsLk9icy9zYCwgZGlnaXRzID0gNiwgc2NpZW50aWZpYyA9IEZBTFNFKQ0KDQpwcmludChydW5fdGltZVsxOihucm93KHJ1bl90aW1lKSAvIDIpXSkNCnByaW50KHJ1bl90aW1lWyhucm93KHJ1bl90aW1lKSAvIDIgKyAxKTpucm93KHJ1bl90aW1lKV0pDQoNCmBgYA0KDQpgYGB7ciBiZW5jaF9zY2FsZTQsIGZpZy5oZWlnaHQ9NiwgZmlnLndpZHRoPTEwfQ0KDQpnZ3Bsb3QocnVuX2FsbCwgYWVzKHggPSBCZW5jaCwgeSA9IFRpbWUsIGNvbG9yID0gRnVuY3Rpb24sIGZpbGwgPSBCZW5jaCkpICsgZ2VvbV9ib3hwbG90KCkgKyBzY2FsZV95X2xvZzEwKGxhYmVscyA9IHNjYWxlczo6Y29tbWEsIGJyZWFrcyA9IGMoMC4wMSwgMC4xLCAxLCAxMCwgMTAwLCAxMDAwLCAxMDAwMCkpICsgc3RhdF9zdW1tYXJ5KGZ1bi55ID0gbWVhbiwgZ2VvbSA9ICJsaW5lIiwgYWVzKGdyb3VwID0gRnVuY3Rpb24pKSArIHN0YXRfc3VtbWFyeShmdW4ueSA9IG1lYW4sIGdlb20gPSAicG9pbnQiLCBhZXMoZ3JvdXAgPSBGdW5jdGlvbikpICsgbGFicyh4ID0gIkJlbmNobWFyayIsIHkgPSAiVGltZSAoTWlsbGlzZWNvbmRzKSIpICsgdGhlbWVfYncoKQ0KDQpgYGANCg0KYGBge3IgYmVuY2hfc2NhbGU1LCBmaWcuaGVpZ2h0PTYsIGZpZy53aWR0aD0xMH0NCg0KZ2dwbG90KHJ1bl9hbGwsIGFlcyh4ID0gQmVuY2gsIHkgPSBNaWxPYnMsIGNvbG9yID0gRnVuY3Rpb24sIGZpbGwgPSBCZW5jaCkpICsgZ2VvbV9ib3hwbG90KCkgKyBzY2FsZV95X2xvZzEwKGxhYmVscyA9IHNjYWxlczo6Y29tbWEsIGJyZWFrcyA9IGMoMSwgMi41LCA1LCA3LjUsIDEwLCAxMi41LCAxNSwgMTcuNSwgMjAsIDIyLjUsIDI1KSwgbGltaXRzID0gYygxLCBOQSkpICsgc3RhdF9zdW1tYXJ5KGZ1bi55ID0gbWVhbiwgZ2VvbSA9ICJsaW5lIiwgYWVzKGdyb3VwID0gRnVuY3Rpb24pKSArIHN0YXRfc3VtbWFyeShmdW4ueSA9IG1lYW4sIGdlb20gPSAicG9pbnQiLCBhZXMoZ3JvdXAgPSBGdW5jdGlvbikpICsgbGFicyh4ID0gIkJlbmNobWFyayIsIHkgPSAiVGhyb3VnaHB1dCAoTWlsbGlvbiBPYnMuL3MpIikgKyB0aGVtZV9idygpDQoNCmBgYA0KDQpgYGB7ciBiZW5jaF9zY2FsZTYsIGZpZy5oZWlnaHQ9NiwgZmlnLndpZHRoPTEwfQ0KDQpnZ3Bsb3QoZGF0YSA9IHJ1bl9hbGwsIGFlcyh4ID0gTWlsT2JzLCBjb2xvciA9IEZ1bmN0aW9uLCBmaWxsID0gRnVuY3Rpb24sIGdyb3VwID0gRnVuY3Rpb24pKSArIGNvb3JkX2ZsaXAoKSArIHN0YXRfZWNkZihhZXMoeW1pbiA9IC4ueS4uLCB5bWF4ID0gMSksIGFscGhhID0gMC41LCBnZW9tID0gInJpYmJvbiIpICsgc3RhdF9lY2RmKGdlb20gPSAibGluZSIsIHNpemUgPSAyLCBhbHBoYSA9IDAuNzUsIHBhZCA9IEZBTFNFKSArIGxhYnMoeCA9ICJUaHJvdWdocHV0IChNaWxsaW9uIE9icy4vcykiLCB5ID0gIlBlcmNlbnRpbGUiKSArIGZhY2V0X3dyYXAofiBCZW5jaCwgZGlyID0gImgiLCBuY29sID0gMiwgc2NhbGVzID0gImZyZWUiKSArIHRoZW1lX2J3KCkNCg0KYGBgDQoNCg==